from langchain_text_splitters import RecursiveCharacterTextSplitter

with open("docs/注意力机制.txt", "r", encoding="utf-8") as f:
    state_of_the_union = f.read()

text_splitter = RecursiveCharacterTextSplitter(
    chunk_size=100, # 每个块的最大大小，大小由 length_function 决定。# 分隔符列表（默认为 ['\n\n', '\n', ' ', '']）
    chunk_overlap=20, # 块之间的目标重叠。重叠的块有助于减轻在块之间划分上下文时信息的丢失。
    length_function=len,# 确定块大小的函数。
    is_separator_regex=False, #分隔符列表（是否应被解释为正则表达式。
    separators=[ # 分隔符列表（默认为 ['\n\n', '\n', ' ', '.']）
        "\n\n",
        "\n",
        " ",
        ".",
        ",",
        "\u200b",  # Zero-width space
        "\uff0c",  # Fullwidth comma
        "\u3001",  # Ideographic comma
        "\uff0e",  # Fullwidth full stop
        "\u3002",  # Ideographic full stop
        "",
    ],
)
texts = text_splitter.create_documents([state_of_the_union])
print(texts[0])
print(texts[1])